{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 采用 xgboost 模型完成商品分类"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 调整正则化参数：reg_alpha 和reg_lambda"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "首先导入必要的工具包"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "import math\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###   数据读取\n",
    "注：数据已经提前被处理，只需根据作业要求读取 RentListingInquries_FE_train.csv 文件即可"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# path to where the data lies\n",
    "path = './data/' \n",
    "fileName = 'RentListingInquries_FE_train.csv'\n",
    "\n",
    "data=pd.read_csv(path+fileName)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 从原始数据中分离输入特征x和输出y\n",
    "X = data.drop('interest_level', axis = 1)\n",
    "y = data['interest_level']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 将数据分割训练数据与测试数据\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# 随机采样20%的数据构建测试样本，其余作为训练样本\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3,random_state = 0)\n",
    "#print('X_train.shape: %s %s'%X_train.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "各类样本不均衡，交叉验证是采用StratifiedKFold，在每折采样时各类样本按比例采样"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# prepare cross validation\n",
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'reg_alpha': [1.5, 2], 'reg_lambda': [0.5, 1, 2]}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#reg_alpha = [1e-3, 1e-2, 0.05, 0.1]    #default = 0\n",
    "#reg_lambda = [1e-3, 1e-2, 0.05, 0.1]   #default = 1\n",
    "\n",
    "reg_alpha = [ 1.5, 2]    #default = 0, 测试0.1,1，1.5，2\n",
    "reg_lambda = [0.5, 1, 2]      #default = 1，测试0.1， 0.5， 1，2\n",
    "\n",
    "param_test5_1 = dict(reg_alpha=reg_alpha, reg_lambda=reg_lambda)\n",
    "param_test5_1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/lib/python3.6/site-packages/sklearn/model_selection/_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.60481, std: 0.00480, params: {'reg_alpha': 1.5, 'reg_lambda': 0.5},\n",
       "  mean: -0.60533, std: 0.00466, params: {'reg_alpha': 1.5, 'reg_lambda': 1},\n",
       "  mean: -0.60352, std: 0.00466, params: {'reg_alpha': 1.5, 'reg_lambda': 2},\n",
       "  mean: -0.60338, std: 0.00577, params: {'reg_alpha': 2, 'reg_lambda': 0.5},\n",
       "  mean: -0.60380, std: 0.00433, params: {'reg_alpha': 2, 'reg_lambda': 1},\n",
       "  mean: -0.60334, std: 0.00416, params: {'reg_alpha': 2, 'reg_lambda': 2}],\n",
       " {'reg_alpha': 2, 'reg_lambda': 2},\n",
       " -0.6033412794227035)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb5_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=645,  #第二轮参数调整得到的n_estimators最优值\n",
    "        max_depth=6,\n",
    "        min_child_weight=4,\n",
    "        gamma=0,\n",
    "        subsample=0.7,\n",
    "        colsample_bytree=0.6,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch5_1 = GridSearchCV(xgb5_1, param_grid = param_test5_1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch5_1.fit(X_train , y_train)\n",
    "\n",
    "gsearch5_1.grid_scores_, gsearch5_1.best_params_,     gsearch5_1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/root/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/root/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/root/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/root/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/root/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/root/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([401.57876554, 413.92272644, 434.78200135, 440.51435671,\n",
       "        412.20908427, 353.34096007]),\n",
       " 'std_fit_time': array([ 0.81592528, 17.99756758,  1.86229267,  2.69986314, 63.17671199,\n",
       "        33.57155179]),\n",
       " 'mean_score_time': array([5.85793509, 3.76822157, 4.80686135, 4.14853415, 3.97651873,\n",
       "        1.64238267]),\n",
       " 'std_score_time': array([0.73943847, 0.52877851, 0.98132021, 0.66721006, 1.06475042,\n",
       "        0.08398626]),\n",
       " 'param_reg_alpha': masked_array(data=[1.5, 1.5, 1.5, 2, 2, 2],\n",
       "              mask=[False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_reg_lambda': masked_array(data=[0.5, 1, 2, 0.5, 1, 2],\n",
       "              mask=[False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'reg_alpha': 1.5, 'reg_lambda': 0.5},\n",
       "  {'reg_alpha': 1.5, 'reg_lambda': 1},\n",
       "  {'reg_alpha': 1.5, 'reg_lambda': 2},\n",
       "  {'reg_alpha': 2, 'reg_lambda': 0.5},\n",
       "  {'reg_alpha': 2, 'reg_lambda': 1},\n",
       "  {'reg_alpha': 2, 'reg_lambda': 2}],\n",
       " 'split0_test_score': array([-0.6115512 , -0.61181465, -0.61021466, -0.61009185, -0.60816709,\n",
       "        -0.60899253]),\n",
       " 'split1_test_score': array([-0.60318874, -0.60272119, -0.60194559, -0.5996448 , -0.6018849 ,\n",
       "        -0.60079747]),\n",
       " 'split2_test_score': array([-0.59776521, -0.59990597, -0.59740584, -0.59707104, -0.59770243,\n",
       "        -0.59759937]),\n",
       " 'split3_test_score': array([-0.60296469, -0.60231122, -0.60059525, -0.59946629, -0.60191443,\n",
       "        -0.60223901]),\n",
       " 'split4_test_score': array([-0.60857089, -0.60992047, -0.60741873, -0.61063185, -0.60931509,\n",
       "        -0.6070781 ]),\n",
       " 'mean_test_score': array([-0.60480819, -0.60533468, -0.60351605, -0.60338104, -0.6037967 ,\n",
       "        -0.60334128]),\n",
       " 'std_test_score': array([0.00480109, 0.00465753, 0.00465695, 0.00577419, 0.00433329,\n",
       "        0.0041607 ]),\n",
       " 'rank_test_score': array([5, 6, 3, 2, 4, 1], dtype=int32),\n",
       " 'split0_train_score': array([-0.2928907 , -0.29626169, -0.30325723, -0.30124911, -0.30398957,\n",
       "        -0.3093407 ]),\n",
       " 'split1_train_score': array([-0.2929582 , -0.29660224, -0.30254501, -0.30111629, -0.30488303,\n",
       "        -0.30934993]),\n",
       " 'split2_train_score': array([-0.2956921 , -0.29827187, -0.3054904 , -0.30406164, -0.30638244,\n",
       "        -0.31200625]),\n",
       " 'split3_train_score': array([-0.29351658, -0.2982058 , -0.30323222, -0.30186816, -0.30432675,\n",
       "        -0.31040191]),\n",
       " 'split4_train_score': array([-0.29403896, -0.29681502, -0.30375783, -0.30148271, -0.30617502,\n",
       "        -0.3113498 ]),\n",
       " 'mean_train_score': array([-0.29381931, -0.29723133, -0.30365654, -0.30195558, -0.30515136,\n",
       "        -0.31048972]),\n",
       " 'std_train_score': array([0.0010248 , 0.00084162, 0.00099481, 0.00108353, 0.00096594,\n",
       "        0.00106458])}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch5_1.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/root/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/root/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/root/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/root/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/root/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/root/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.603341 using {'reg_alpha': 2, 'reg_lambda': 2}\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZ4AAAELCAYAAADp1+D/AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl4VdXV+PHvIgHCHCBMISCDiUxCgJDKZAELoqK89acIbRVQQV6HqrT2ra1WXvStthWrVRRRQLCKoq2IKGKsoIyGMAphDihhJsyQhAzr98c5Vy4hw4XkDknW53nuQ+45+5y7DpCs7HP22ltUFWOMMSZQqgQ7AGOMMZWLJR5jjDEBZYnHGGNMQFniMcYYE1CWeIwxxgSUJR5jjDEBZYnHGGNMQFniMcYYE1CWeIwxxgRUeLADCEVRUVHaqlWrYIdhjDHlyurVq4+oaqOS2lniKUSrVq1ISUkJdhjGGFOuiMj3vrSzW23GGGMCyhKPMcaYgLLEY4wxJqDsGY8xJiTl5OSQnp5OVlZWsEMxBURERBATE0PVqlUv63hLPMaYkJSenk6dOnVo1aoVIhLscIxLVcnIyCA9PZ3WrVtf1jnsVpsxJiRlZWXRsGFDSzohRkRo2LBhqXqilniMMSHLkk5oKu2/iyUeU+Gl7jvJt2kZwQ7DGOOyZzymwtp28BR/T9rGgo0HAHjipvbc27dNkKMyxliPx1Q4u46c4eH31nL9i9+wZPsRHr4ulhs6NeWZTzfz9PxU8vM12CEac5HFixczZMiQUrcpzpYtW+jZsyfVq1fn+eefL7LdqFGjaN26NfHx8cTHx7Nu3brL/szCWI/HVBh7jp7l5a+28681e6kWVoVxP23L2L5tqF+rGnn5ytPzU5m2dBcHTmbxwrAuVA8PC3bIphxRVVSVKlXK7+/rDRo04B//+Adz584tse3f/vY3brvtNr/EYYnHlHsHTmTxyqLtvL9qDyLCqF6tGPfTtjSqU/3HNmFVhKdu7kB0ZAR//mwLR05lM/WuBOrVuLw6BBNY//vJJlL3nSzTc3aIrstTN3csts3u3bu54YYb6N+/PytWrOCRRx5hypQpZGdn07ZtW2bMmEHt2rX57LPPGD9+PFFRUXTr1o20tDTmz59f6DmTk5N55JFHyMzMpEaNGsyYMYOrrrrqgjYTJkxg586d7N27lz179vC73/2OMWPGAHD69Gluu+02Nm7cSPfu3fnnP/+JiDBx4kQ++eQTMjMz6dWrF6+//vpFgwAaN25M48aN+fTTT0vxN1d65Td1m0rvyOlsnp6fyrV/W8T7q/ZwR48WfPNYf54c0uGCpOMhIoy9ti0vDY9nzQ/HuH3KcvYdzwxC5KY82bp1K3fddRdJSUlMmzaNL7/8kjVr1pCQkMALL7xAVlYW9913HwsWLGDp0qUcPny42PO1a9eOb775hrVr1zJx4kT+8Ic/FNpuw4YNfPrpp6xYsYKJEyeyb98+ANauXcuLL75IamoqaWlpLFu2DIAHH3yQVatWsXHjRjIzM39MfFOmTGHKlCmXfN1//OMf6dy5M48++ijZ2dmXfHxx/NrjEZHBwEtAGPCmqj5XSJthwARAgfWq+gt3+0jgCbfZM6o6092+GGgGeH5iDFLVQyIyHrgXyAUOA3er6vfuMXnAd277H1T1ljK+VBNAx8+e4/Vv0nhr2W7O5eXz/7o156EBsbRoUNOn44fGN6dR7erc9/Zqbn11OW/d3YN2Tev6OWpTGiX1TPzpiiuu4JprrmH+/PmkpqbSu3dvAM6dO0fPnj3ZsmULbdq0+bGYcsSIEUydOrXI8504cYKRI0eyfft2RIScnJxC2w0dOpQaNWpQo0YN+vfvT3JyMpGRkSQmJhITEwNAfHw8u3fvpk+fPixatIi//vWvnD17lqNHj9KxY0duvvlmxo0bd8nX/Oyzz9K0aVPOnTvH2LFj+ctf/sKf/vSnSz5PUfyWeEQkDJgMDATSgVUiMk9VU73axAKPA71V9ZiINHa3NwCeAhJwEtJq99hj7qG/VNWC6xasBRJU9ayI/DfwV+AOd1+mqsb750pNoJzMymH60l1MW7KL0+dyuaVLNA9fF0ubRrUv+Vy9roxizriejJqRzO1TVjD1zgR6tm3oh6hNeVerVi3AecYzcOBAZs+efcH+tWvXXtL5nnzySfr3789HH33E7t276devX6HtCt4m87yvXt3rFnJYGLm5uWRlZXH//feTkpJCixYtmDBhQqkKPJs1a/bjZ40ePbrYgQiXw5+32hKBHaqapqrngPeAoQXajAEmexKKqh5yt18PJKnqUXdfEjC4uA9T1UWqetZ9uxKIKaPrMEF29lwury7eQd+/LOLFL7fTJzaKhY9cy0vDu15W0vFo36wu/76/N03qRjByejKfrN9XhlGbiuaaa65h2bJl7NixA4CzZ8+ybds22rVrR1paGrt37wbg/fffL/Y8J06coHnz5gC89dZbRbb7+OOPycrKIiMjg8WLF9OjR48i23qSTFRUFKdPn+bDDz+8hCu72P79+wEn2c6dO5dOnTqV6nwF+fNWW3Ngj9f7dOAnBdrEAYjIMpzbcRNU9fMijm3u9X6Ge/vsXzi34QqOj70HWOD1PkJEUnBuwz2nqiUP6TBBl5WTxzvf/sBri3dw5PQ5BrRrzPiBcXRqXq/MPqN5ZA0+HNeTsbNW89DstRw8mWW1PqZQjRo14q233mLEiBE/PvN45plniIuL49VXX2Xw4MFERUWRmJhY7Hl+97vfMXLkSF544QUGDBhQZLvExERuuukmfvjhB5588kmio6PZtm1boW0jIyMZM2YMV199Na1atbogSXme74wbN44DBw6QkJDAyZMnqVKlyo/PiurWrcuNN97Im2++SXR0NL/85S85fPgwqkp8fPxlPSMqjlz8M7uMTixyO3C9qt7rvr8TSFTVh7zazAdygGE4PZQlQCecnlB1VX3GbfckcFZVJ4lIc1XdKyJ1cBLPP1V1ltc5fwU8CPxUVbPdbdGquk9E2gBfAdep6s4C8Y4FxgK0bNmy+/ff+7SQnvGDc7n5vJ+yh1e+2s7Bk9n0uTKK8YPi6Nayvt8+Mysnj/Fz1vHZdwe4p09r/nhje6pUselagmnz5s20b98+2GH45PTp09SuXRtV5YEHHiA2NpZHH330ss83YcIEateuzW9/+9syjLJsFfbvIyKrVTWhpGP92eNJB1p4vY8BCt7LSAdWqmoOsEtEtgKx7vZ+BY5dDKCqe90/T4nIuzi39GYBiMjPgD/ilXTctvvcP9PcwQldgQsSj6pOBaYCJCQkWIVhEOTm5fPvtXt56cvt7D2eSY9W9Xnxjq4BefYSUTWMl0d0o3Edq/Uxl+6NN95g5syZnDt3jq5du3LfffcFO6SQ5s8eTziwDbgO2AusAn6hqpu82gwGRqjqSBGJwhkgEI87oADo5jZdA3QHTgKRqnpERKoCs4EvVXWKiHQFPgQGq+p2r8+oj9NbynY/YwUw1HuQQ0EJCQmaklJw7ILxl7x8Zf6Gfbz45XZ2HTlDl5h6/GbQVfSNjQr4JJGqyhtL0vjzZ1v4SesGVusTROWpx1OYGTNm8NJLL12wrXfv3kyePDlIEZWtkOzxqGquiDwILMR5fjNdVTeJyEQgRVXnufsGiUgqkAc8pqoZ7gU8jZOsACaq6lERqQUsdJNOGPAl8Ibb5m9AbeAD94eVZ9h0e+B1EcnHGUzxXHFJxwSOqrJw0wFeSNrGtoOnad+sLm/clcDP2jcO2qzEnlqfJnUj+O0H67l9ynLeGp1IdGSNoMRjyq/Ro0czevToYIcRkvzW4ynPrMfjX6rKoq2HmPTFNjbtO0nbRrUYP/AqbujUNKSeqyzfcYT73l5NrerhVusTBOW9x1PRlabHYzMXmIBRVZZuP8Ktry3n7rdSOJ2dywvDuvDFoz/lps7NQirpwPlaH0W5fcoKVuy0pRWMKQuWeExAJO86yvCpK/nVtG85eCKL5269mi/H/5Rbu8UQFmIJx5vV+hhT9mySUONX6/ccZ1LSNr7ZdphGdarzv7d0ZHhii3I1WsxqfYwpW9bjMX6Ruu8k985MYejkZWzce4I/3tiebx7rz8hercpV0vGIrFmNWfckcuPVtq6P8Y9ArMfzzjvv0LlzZzp37kyvXr1Yv379ZZ+rNKzHY8rUjkOn+HvSdj79bj91I8L57aA4RvVuTe3q5f+/mtX6VG4VYT2e1q1b8/XXX1O/fn0WLFjA2LFj+fbbbwMeR/n/aWBCwvcZZ3jpy+3MXbeXGlXD+PWAK7mnb5sKVwNj6/oEyYLfw4HvSm53KZpeDTdcNGH+BSraejy9evX68etrrrmG9PT0y/mbK7Xym7pNSNh7PJPf/2sDAyZ9zWcb9zOmbxuW/M8Axg+6qsL+MLZ1fSqXiroez7Rp07jhhhtK81dz2azHYy7LoZNZTF60g9nJzlyud15zBff3b0vjOhFBjixwbF2fACqhZ+JPFXE9nkWLFjFt2jSWLl1a2r+ey2KJx1ySjNPZTPl6J7NWfE9evnJ7QgseGnBlpa3st3V9Kr6Kth7Phg0buPfee1mwYAENGwbn/6rdajM+OXE2h+cXbqXvXxcxbekuhnSO5qvf9OPZW6+utEnHw1Pr09RqfSq0irAezw8//MCtt97K22+/TVxcXLFx+pP1eEyxTmXlMGPZbt5YksaprFxudlf9vLLx5S/AVhE5tT69GDMrxWp9KqiKsB7PxIkTycjI4P777wcgPDycYEwPZnO1FcLmaoPMc3nMWrGbKV/v5NjZHAZ1aMKjA+No38yeYRTHe12fu3u35ombbF2fy1We5mqz9XgcQZ+d2pRPWTl5zE7+gcmLdnLkdDb9rmrE+IFxdI6JDHZo5UJE1TBeGdGNp+umMn3ZLg6eymLS7V2IqGq1PhWZrcdzaazHU4jK2OPJycvng5R0Xv5qO/tPZNGzTUN+MyiOhFYNgh1auaSqvLlkF//32WYSWzfgjTsTqFezYg4v95fy1OMpjK3HUzTr8VRyuXn5zF23j5f+s409RzPp1jKSSbd3odeVUcEOrVwTEcZc24bGdas76/q8buv6XA5VDdraTKVVkdfjKW2HxRJPJZWfr8z/bj8vfrmNtMNn6NS8LhNHd6JfXKNy+40eiqzW5/JFRESQkZFBw4YN7f9kCFFVMjIyiIi4/Jo9v95qc5e2fglntdA3VfWiKjARGQZMwFnuer2q/sLdPhJ4wm32jKrOdLcvBpoBnlLxQap6SESqA7NwlsjOAO5Q1d3uMY8D9+CscvprVV1YXNwV+VabqvJF6kH+nrSNLQdOcVWTOowfFMegDk3sm9uPNu8/yagZyZzNzuP1u7rTq631KEuSk5NDenp6kfUoJngiIiKIiYmhatULbx/7eqvNb4lHRMKAbcBAIB1nGesR3stOi0gsMAcYoKrHRKSxm0QaAClAAk5CWg10d9ssBn6rqikFPu9+oLOqjhOR4cDPVfUOEekAzAYSgWic5bLjVDWvqNgrYuJRVRZvO8zfk7axIf0EbaJq8cjAOIZcHXoLsFVUe49nMmp6Mt9nnOX5YV24pUt0sEMypkyFwgqkicAOVU1T1XPAe8DQAm3GAJNV9RiAqh5yt18PJKnqUXdfEjC4hM8bCsx0v/4QuE6cX+GHAu+paraq7gJ2uLFVGst3HuG2KSsYPWMVR8+c4/nbu/DFo9dyS5doSzoB5Kn1iW8Rya9nr+XNJWnBDsmYoPDnM57mwB6v9+nATwq0iQMQkWU4t+MmqOrnRRzb3Ov9DBHJA/6FcxtOvY9R1VwROQE0dLevLOZcFdbq748y6YttLN+ZQdO6Efzfzztxe/cWVAu3CSuCpV7Nqsy6J5Hxc9bxzKeb2Xc8y2p9TKXjz8RT2HdSwft64UAs0A+IAZaISKcSjv2lqu4VkTo4iedOnGc7RR3jSxyIyFhgLEDLli0LOaT8+C79BJOStrJ462GialfnqZs7MCKxpdWShAir9TGVnT8TTzrQwut9DFBwEqt0YKWq5gC7RGQrTiJKx0lG3scuBlDVve6fp0TkXZzbZrO8Pi9dRMKBesBRH+NAVacCU8F5xnPJVxsCthw4yd+TtrFw00Eia1bl9ze0466eV1Czmg1eDDVVqgh/GtKB6Ho1+L/PNnP4VLbV+phKw5/3XFYBsSLSWkSqAcOBeQXazAX6A4hIFM6ttzRgITBIROqLSH1gELBQRMLddohIVWAIsNE91zxgpPv1bcBX7i24ecBwEakuIq1xEluyX644SHYePs1Ds9dyw0tLWL4jg/ED41jyu/6M+2lbSzohzFPr89LweNb+cIzbX7d1fUzl4LefSu5zlgdxkkgYMF1VN4nIRCBFVedxPsGk4gx1fkxVMwBE5Gmc5AUwUVWPikgtnARU1T3nl8AbbptpwNsisgOnpzPcjWOTiMwBUoFc4IHiRrSVJz9knOWl/2zno7XpRFQN4/5+bRnTtw2RNasFOzRzCazWx1Q2NmVOIUJ9OPX+E5m8/NUO5qzaQ1gV4c5rrmBcv7ZE1a5e8sEmZFmtjynvgl7HU56FauI5dCqLVxft5N3kH1BVRiS25IH+V9KkbuVZ9bOis1ofU57ZXG0VyLEz55jyzU5mLt9NTp5ye/cYHhxwJTH1awY7NFPGvNf1+fXstRyydX1MBWSJJ4SdyMxh2pI0pi/bzZlzufxXfHMevi6WVlG1gh2a8SOr9TEVnSWeEHQmO5e3lu/m9a93cjIrl5uubsYjP4sltkmdYIdmAsRqfUxFZoknhGTl5PH2iu957eudHD1zjp+1b8yjA+PoGF0v2KGZILBaH1NRWeIJAdm5eby/ag+vfLWDQ6ey6RsbxfiBcXRtWT/YoZkgs3V9TEVkiSeIcvLy+dfqdF7+agd7j2eS2KoBL4/oyk/aNAx2aCbEWK2PqUhsOHUh/D2cOi9fmbd+Ly9+uZ3vM84S3yKS3wyKo8+VUbYmjimW1fqYUBYKyyKYAvLzlU837Of6F7/h0ffXU6taONNGJvDR/b3oG2srf5qStW9Wl3/f35um9SIYNX0V89ZfNO2gMSHPbrUFgKryn82HmJS0jc37TxLbuDav/bIb13dsakNkzSWzWh9T3lni8SNVZcn2I0xK2sb6Pcdp1bAmL94Rz81dogmzhGNKwWp9THlmicdPvk3LYNIX20jefZTmkTX46//rzK3dmhMeZnc3TdmwWh9TXlniKWNrfjjGC19sY+mOIzSuU52nh3ZkWI8WVA+3Hwam7FWpIjx1c0eaR9bgmU+t1seUD5Z4ylDa4dPc+upyGtaqxhM3tedX11xhv32agLi3bxsa143gN3PWWa2PCXk2nLoQpRlOPX/DPvpf1Zha1S2nm8BbvvMI981aTa3q4VbrYwLOhlMHyZDO0ZZ0TND0ahvFB//dE4DbX1vB8p1HghyRMRfza+IRkcEislVEdojI74toM0xEUkVkk4i867V9pIhsd18jCzlunohs9Hr/voisc1+7RWSdu72ViGR67Zvij2s1JlS0a1qXf9/fi2aRVutjQpPffjUXkTBgMjAQSAdWicg8VU31ahMLPA70VtVjItLY3d4AeApIABRY7R57zN1/K3Da+/NU9Q6v804CTnjt3qmq8X64TGNCUnRkDT64rxdj3nZqfQ6eyOLevq2tSNmEBH/2eBKBHaqapqrngPeAoQXajAEmexKKqh5yt18PJKnqUXdfEjAYQERqA+OBZwr7UHG+s4YBs8v4eowpV+rVrMqsuxO56epm/N9nm5k4P5X8fHuma4LPn4mnObDH6326u81bHBAnIstEZKWIDPbh2KeBScDZIj63L3BQVbd7bWstImtF5GsR6VvYQSIyVkRSRCTl8OHDJV6cMeVBRNUwXh7Rlbt7t2bGst08NHstWTl5wQ7LVHL+fApeWJ++4K9b4UAs0A+IAZaISKeijhWReOBKVX1URFoV8bkjuLC3sx9oqaoZItIdmCsiHVX15AUnV50KTAVnVFtxF2ZMeVKlivCnmzsQHRnh1PqctlofE1z+7PGkAy283scABZ9ypgMfq2qOqu4CtuIkoqKO7Ql0F5HdwFKc3tJiTyMRCQduBd73bFPVbFXNcL9eDezE6WkZU6nc27cN/xjRlbU/HOO2KcvZdzwz2CGZSsqfiWcVECsirUWkGjAcmFegzVygP4CIROEkhDRgITBIROqLSH1gELBQVV9T1WhVbQX0Abapaj+v8/0M2KKq6Z4NItLIHeiAiLTBSWxpZX61xpQDt3SJZubdiRw4kcXPX13G5v0nSz7ImDLmt8SjqrnAgzhJZDMwR1U3ichEEbnFbbYQyBCRVGAR8JiqZqjqUZxnOavc10R3W0mGc/GggmuBDSKyHvgQGOfjuYypkDy1PoIwbIrV+pjAs5kLCuHvheCMCQX7jmcyakYyu46cYdKweG7pEh3skEw5ZzMXGGOK5an16dqyPr+evZY3vknDfhE1gWCJx5hKzGp9TDDYpGLGVHKeWp8mdSOYvmwXh05mM2mYretj/McSjzHGan1MQNmtNmPMj6zWxwSCJR5jzAWs1sf4myUeY8xFrNbH+JMlHmNMobzX9Rk5PdnW9TFlxhKPMaZIVutj/MESjzGmWFbrY8qaDac2xpTIan1MWbLEY4zxidX6mLJit9qMMZfEan1MaV1S4hGRKiJS11/BGGPKB6v1MaVRYuIRkXdFpK6I1AJSga0i8pj/QzPGhDKr9TGXy5ceTwdVPQn8F/AZ0BK405eTi8hgEdkqIjtE5PdFtBkmIqkisklE3vXaPlJEtruvkYUcN09ENnq9nyAie0Vknfu60Wvf424MW0Xkel9iN8aUzGp9zOXwZXBBVRGpipN4XlHVHBEpcSylu9z0ZGAgkA6sEpF5qprq1SYWeBzorarHRKSxu70B8BSQACiw2j32mLv/VuB0IR/7d1V9vkAcHXBWJu0IRANfikicqub5cO3GmBJ4an3Gvp3Cr2ev5eCJLO7t2xoRCXZoJkT50uN5HdgN1AK+EZErAF9u6CYCO1Q1TVXPAe8BQwu0GQNM9iQUVT3kbr8eSFLVo+6+JGAwgIjUBsYDz/gQA+5nvqeq2aq6C9jhxmaMKSP1alZl1j2J3NTZan1MyUpMPKr6D1Vtrqo3quN7oL8P524O7PF6n+5u8xYHxInIMhFZKSKDfTj2aWAScLaQz3xQRDaIyHQRqX8JcRhjSql6eBgvD+/KPX1aM2PZbh6avZasHLuxYC7my+CCh93BBSIi00RkDTDAh3MX1s8u+CtQOBAL9ANGAG+KSGRRx4pIPHClqn5UyP7XgLZAPLAfJzn5GgciMlZEUkQk5fDhw4UcYowpSZUqwpNDOvDETe359Lv93DU9mRNnc4Idlgkxvtxqu9sdXDAIaASMBp7z4bh0oIXX+xig4JPHdOBjVc1xb4NtxUlERR3bE+guIruBpTi9pcUAqnpQVfNUNR94g/O303yJA1WdqqoJqprQqFEjHy7PGFMUT63Puh+Oc9uU5ey1Wh/jxZfE4+kx3AjMUNX1FN6LKGgVECsirUWkGs4D/nkF2szFvW0nIlE4t97SgIXAIBGp794yGwQsVNXXVDVaVVsBfYBtqtrPPb6Z13l/DnhGvM0DhotIdRFpjZPYkn2I3xhTCj/W+pzM4lar9TFefEk8q0XkC5zEs1BE6gD5JR2kqrnAgzhJZDMwR1U3ichEEbnFbbYQyBCRVGAR8JiqZqjqUZxnOavc10R3W3H+KiLficgGnGT2qBvHJmAOTg3S58ADNqLNmMDo2bYhH4zzqvXZYbU+BqSkKc5FpArOc5M0VT0uIg2B5qq6IRABBkNCQoKmpKQEOwxjKox9xzMZNSOZXUfO8PztXRgab+N7KiIRWa2qCSW182VUWz7Oc5EnROR5oFdFTjrGmLIXHVmDD8b1olvL+jz83jqmfrPT1vWpxHwZ1fYc8DDOrapU4Nci8qy/AzPGVCz1apyv9fnzZ1v4309SybNan0rJl5kLbgTi3Z4PIjITWIsz44AxxvjMU+vTtG4E05bu4tCpLF4YFm/r+lQyvs5OHen1dT1/BGKMqRy8a30+++4Ad02zWp/KxpfE8yywVkTecns7q4E/+zcsY0xF92Otzx6r9alsfBlcMBu4Bvi3++oJfOPnuIwxlYDV+lROPt1qU9X9qjpPVT9W1QPASj/HZYypJKzWp/K53KWvbb5zY0yZuWBdnxnJfLxub7BDMn50uYnHxkAaY8qU1fpUHkUOpxaRlyk8wQgXjnIzxpgy4an1GT9nPX/+bAv7jmfx5JAOhFWxmywVSXF1PMXNGWPzyRhj/MJqfSq+IhOPqs4MZCDGGOPhqfVpVi+CZz7dzJFTybxxVwL1alYNdmimDFzuMx5jjPE7q/WpmCzxGGNCmtX6VDyWeIwxIc9qfSoWX9bj+Uchm08AKar6sV+iCjJbj8eY0GTr+oS2MluPB4jAWQhuu/vqDDQA7hGRF0sIYrCIbBWRHSLy+yLaDBORVBHZJCLvem0fKSLb3dfIQo6bJyIbvd7/TUS2iMgGEflIRCLd7a1EJFNE1rmvKT5cszEmBFmtT8Xgy7IIVwID3KWsEZHXgC+AgcB3RR0kImHAZLddOrBKROapaqpXm1ic5RV6q+oxEWnsbm8APAUk4NQSrXaPPebuvxU4XeAjk4DHVTVXRP7invd/3H07VTXeh2s1xoQ4q/Up/3zp8TQHanm9rwVEq2oekF3McYnADlVNU9VzwHvA0AJtxgCTPQlFVQ+5268HklT1qLsvCRgMICK1gfHAM94nUtUvPMkRZy65GB+uzRhTDnlqfe7p05q3lu/modlryMrJC3ZYxke+JJ6/AutEZIaIvIWzCNzzIlIL+LKY45oDe7zep7vbvMUBcSKyTERWishgH459GpgEnC3ms+8GFni9by0ia0XkaxHpW8xxxphywtb1Kb98WRZhGtALmOu++qjqm6p6RlUfK+bQwvq9BW/GhgOxQD9gBPCm+2ym0GNFJB64UlU/KvJDRf4I5ALvuJv2Ay1VtStOT+ldEalbyHFjRSRFRFIOHz5czGUZY0KJ1fqUP74Op+4B9AX6AN19PCYdaOH1PgbYV0ibj1U1R1V3AVtYbjSWAAAevElEQVRxElFRx/YEuovIbmApTm9psaeROwhhCPBLdZ84qmq2qma4X68GduL0tC6gqlNVNUFVExo1auTjJRpjQoHV+pQvJSYeEXkOeBhIdV+/FpFnfTj3KiBWRFqLSDVgODCvQJu5QH/3c6JwEkIasBAYJCL1RaQ+MAhYqKqvqWq0qrbCSYLbVLWfe/xgnMEEt6jqj7fhRKSRO9ABEWmDk9jSfIjfGFOOWK1P+eFLj+dGYKCqTlfV6TgP+W8q6SD3Qf+DOElkMzBHVTeJyEQRucVtthDIEJFUYBHwmKpmqOpRnGc5q9zXRHdbcV4B6gBJBYZNXwtsEJH1wIfAOB/OZYwph9o1rctHD/QiOrKGresTwnwpIN0A9PP8sHaHOi9W1c4BiC8orIDUmPLtRGYOY2el8O2uo/zhxnaM6dsGERtu7W9lWUD6LLBWRN4SkZnAauDPpQ3QGGP8xVPrc1PnZvz5sy387yep5OVboWmoKLGAVFVnuw/we+CMNvsfbI43Y0yI89T6NKsbwZtLd3HwZBZ/v8PW9QkFvsxcgKrux2tggIj8ALT0V1DGGFMWqlQRnhjSgabuuj4Zp5OZeld3ImtWC3Zoldrl9lzsZqkxpty4t28bXv6x1meF1foE2eUmHrtZaowpV252a30OurU+qfus1idYirzVJiIvU3iCESDSbxEZY4yf9GzbkA/H9WLk9GSGvb6C1+/sTu8ro4IdVqVTXI8nBWcEW8FXCvCQ/0Mzxpiyd1XTOnz0QC+aR9ZglNX6BEWRPR5VnVlwm4g0VdUD/g3JGGP8q1m9GswZ15Oxs1J4+L11HDiRxdhrrdYnUC71Gc9nfonCGGMCzLvW59kFVusTSD4Np/Zivw4YYyoMq/UJjktNPG/4JQpjjAkSq/UJvEu61aaqr/orEGOMCSar9Qkcm/rGGGNcVusTGJZ4jDHGi6fWRxCGvb6CZbauT5mzxGOMMQVYrY9/WeIxxphCeGp9urWsz8PvreP1r3dS0vplxjd+TTwiMlhEtorIDhH5fRFtholIqohsEpF3vbaPFJHt7mtkIcfNE5GNXu8biEiS2z7JXTIbcfzDjWGDiHTzx7UaYyoeq/XxD78lHhEJAyYDNwAdgBEi0qFAm1jgcaC3qnYEHnG3NwCeAn4CJAJPeRKJu/9W4HSBj/w98B9VjQX+477H/fxY9zUWeK0ML9MYU8F5an3u7dOat5bv5sF315CVkxfssMo1f/Z4EoEdqpqmqueA94ChBdqMASar6jEAVT3kbr8eSFLVo+6+JGAwgIjUBsYDzxQ411DAM83PTOC/vLbPUsdKIFJEmpXVRRpjKj5Prc8TN7VnwcYD3DUtmeNnzwU7rHLLn4mnObDH6326u81bHBAnIstEZKWIDPbh2KeBScDZAudq4i5Y51m4rvElxGGMMSWyWp+y4c/EU9j0OgVvjobj3ALrB4wA3hSRyKKOFZF44EpV/aiM40BExopIioikHD58+BJOb4ypTKzWp/T8mXjSgRZe72OAfYW0+VhVc1R1F7AVJxEVdWxPoLuI7AaW4vSWFrttDnpuobl/em7b+RIHqjpVVRNUNaFRo0aXeKnGmMrEan1Kx5+JZxUQKyKtRaQaMByYV6DNXKA/gIhE4dx6SwMWAoNEpL47qGAQsFBVX1PVaFVtBfQBtqlqP/dc8wDP6LeRwMde2+9yR7ddA5zw3JIzxpjLZbU+l89viUdVc4EHcZLIZmCOqm4SkYkicovbbCGQISKpwCLgMVXNUNWjOM9yVrmvie624jwHDBSR7cBA9z04SzmkATtwJjm9v8wu0hhTqVmtz+UR+0u6WEJCgqakpAQ7DGNMOZGdm8f4Oev5dMN+RvVqxZNDOhBWpfKtIiMiq1U1oaR2l7osgjHGmAJsXZ9LY4nHGGPKgK3r4zubq80YY8rQvX3b8MovrNanOJZ4jDGmjA3pHM2se6zWpyiWeIwxxg+uaePU+lQRq/UpyBKPMcb4yVVN6/Dv+63WpyBLPMYY40eeWp/uVzi1PlOs1scSjzHG+Fu9GlWZeXciQzo34zlb18eGUxtjTCBUDw/jH8O70qxeBG8s2cWBE1m8OLxy1vpYj8cYYwKkShXhjzd14MkhHViYeoA7p31bKdf1scRjjDEBdk+f1rw8oivr95zgtikrSD9WcHmxis0SjzHGBMGFtT7LK1WtjyUeY4wJEk+tT1iVylXrY4nHGGOCqGCtz9y1Fb/Wx0a1mYorLxe2fwGrZ8CJdIjuBi16QEwiNGoHVez3LhMaPLU+972dwiPvr+PAySzuu7YNIhVzaQVLPKbiObkP1rwNa2bCyb1Quyk07QRbP4N1/3TaVK8LzbtDi0QnEcUkQI3I4MZtKjVPrc9v5qznuQVbOHAiq8Ku6+PXxCMig4GXgDDgTVV9rpA2w4AJgALrVfUX7vaRwBNus2dUdaa7/XOgmRv7EuABVc0TkfeBq9z2kcBxVY0XkVY4K6BudfetVNVxZXypJtjy8yFtEaRMh60LQPOg7QC44S8QNxjCqoIqZOyE9GTYkwzpq+Cbv4HmO+eIuup8j6hFovPeekUmgCpLrY/fViAVkTBgG84y1Ok4S1iPUNVUrzaxwBxggKoeE5HGqnpIRBoAKUACTkJaDXR329RV1ZPi9EE/BD5Q1fcKfPYk4ISqTnQTz3xV7eRr7LYCaTly5gis/adzO+3YbqjZELr+CrqPggZtSj4++xTsXQ17VjkJKX0VZB5z9lWvBzHd3UTUA5pbr8gEzrSlu3jm01QSrqjPG3cllIt1fUJhBdJEYIeqprkBvQcMBVK92owBJqvqMQBVPeRuvx5IUtWj7rFJwGBgtqp6xhyGA9VwEtOP3IQ0DBjgj4syIUAVvl/u9G5SP4b8HLiiNwx4EtrfDOHVfT9X9TrQpp/z8pw7Y4fbI0p2EtLXf8H5bybQ6CqI6XH+Fl1UnPWKjF/c06c1TepWZ/z767ltygreGt2DmPo1gx1WmfBn4mkO7PF6nw78pECbOAARWYZzO26Cqn5exLHNPW9EZCFOYluA0+vx1hc4qKrbvba1FpG1wEngCVVdUjBYERkLjAVo2bKlj5doAirzGKx/30k4R7Y6PZIe90D30dC4Xdl8hghExTqvrr90tmWddHpF6auc15b5sPZtZ19EPacn1CLRSUgxCc42Y8rAkM7RRNWuzphZKdz66nLeGp1Ih+i6wQ6r1PyZeAp7Ilbwvl44EAv0A2KAJSLSqaRjVfV6EYkA3sHp2SR5tRsBzPZ6vx9oqaoZItIdmCsiHb16Tp5zTgWmgnOrreTLMwGhCnvXOMlm478gN9MZFDB0MnS8FaoF4DfAiLrQtr/z8sRUsFe0+DnO94raXfisqGGs9YrMZfPU+oyakcyw11fw+p3d6X1lVLDDKhV/Jp50oIXX+xhgXyFtVqpqDrBLRLbiJKJ0nGTkfexi7wNVNUtE5uHcvksCEJFw4Fagu1e7bCDb/Xq1iOzE6WnZQ5xQln0avvvASTgHNkDVWtDlDqd3Ex0f3NhK6hXtSYbUebBmlrMvop7bG/J6VhRR/n9rNYHjqfUZNX0Vo2Yk87fbuvBfXZuXfGCI8ufggnCcwQXXAXtxBhf8QlU3ebUZjDPgYKSIRAFrgXjODyjo5jZdg5NMzgF1VHW/e/53gCWq+orX+R5X1Z96fUYj4Kg78q0Nzki4qz3PjwpjgwuC6MBGJ9lsmAPnTkHjjtDjbrh6WPn6YZ2f7/SKvEfQHdrMj72ixu29nhX1sF6R8cmJzBzuezuFlWlH+f0N7UKu1ifogwtUNVdEHgQW4jy/ma6qm0RkIpCiqvPcfYNEJBXIAx5T1Qz3Ap7GSVYAE1X1qIg0AeaJSHX3nF8BU7w+djgX3mYDuBaYKCK57meMKy7pmCDIyYRNc52Ek54MYdWh062QcLfzQzmEvrF8VqUKNIpzXl1/5WzLOnHhCLrUuU6tEUBEpPN8yHpFphgVpdbHbz2e8sx6PAFyZDukzIB170DWcWh4pZNsuoyAmg2CHZ3/5edDxvYLnxUd3kLhvaJE59ZeeUzCpszl5yvPLtjMG0t2Mbhj05Cp9fG1x2OJpxCWePwo95wzKixlOuxeAlXCnSHQCXdDq772gzXrBKSnnH9WlJ4C2SecfRGRF96ea97dekWVXKjV+ljiKQVLPH5wbDesnukMQz5zGCJbOkWe8b+COk2CHV3oys+HI9sufFZ0eIu7U6BxhwIj6K605F3JzN+wj/Hvr6dlw5pBr/WxxFMKlnjKiGeSzpTpsONL5wdi3GCnd9N2AFQJ/q2BcinzOOxN8ZptYfX5XlGN+gVG0HV3imRNhbYyLYMxs1KoUTUsqLU+lnhKwRJPKRU2SWf3kdDtLqgXE+zoKp78fKeg1vOsKD3lfK9Iqji9Iu9nRQ3bWq+oAtp64BSjZiRzKis3aLU+lnhKwRLPZShqks6Eu89P0mkC56JeUQpkuzXTNRq4icjtGTXvDtVrBzdeUyb2n8hk1PRVpB05HZRaH0s8pWCJ5xKUdpJOExgFe0V7Vjnvwe0VdbzwWVGDNtYrKqeCWetjiacULPGUwHuSzs3zIO+cM0lnwt2XPkmnCZ7MY87zIc/Ahb2rz/eKajZ0nxW5t+iiu1mvqBzJzs3jN3PWM3/Dfkb1ahWwWp+gF5CaCijzOKx/78JJOhPuLttJOk3g1KgPsT9zXgD5eXB46/keUXoybPvc2SdVoEnH8z2imB7WKwphob6uj/V4CmE9Hi9FTdKZcHfgJuk0wXP2qDvbQvL5EXTnTjn7rFdULkxfuounA1TrY7faSsESD4VP0tn59tCYpNMET36eM2LOU1O0J9mZfQFAwqBJB+sVhaBPN+zn0ffX+b3WxxJPKVTqxFNRJuk0gXP2qDvbgtezonOnnX01owqMoOsG1WoFN95KamVaBmNnpRBRNYwZo3vQMbrs142yxFMKlS7xVMRJOk3w5Oc5M3F7PyvK2OHskzDnWZGnpqhFD6jf2v6PBci2g6cYOd2p9Znyq+70iS3bWh9LPKVQaRJPZZ+k0wTO2aNe888lO88NPb2iWo0KPCvqar0iP9p/IpPRM1ax49Bp/nZ7Z37eteyKui3xlEKFTjy552Drp07vZtc3NkmnCY78PDiUev5ZUfqqC3tFTTtd+Kyofiv7v1mGTmblcN+s1axIy+B/Brdj3E/LptbHEk8pVMjEc+x7ZwqbNW/DmUPnJ+nseifUbhzs6IzxoVeUeP5ZUXRXG1FZStm5efz2gw18sn4fI3tewZ9u7ljqWh+r4zHOb5WeSTq3J9kknSa01WwAcdc7L7i4V7Qn2emtg9NTb9LpwmdFkVdYr+gSVA8P46U74mlWL4Kp36Rx4GQWLw3vGpBaH7/2eNylqF/CWS30TVV9rpA2w4AJOKtfrVfVX7jbRwJPuM2eUdWZ7vbPgWY4SXMJ8IC7rPUEYAxw2D3mD6r6mXvM48A9OCuQ/lpVFxYXd7nv8Zzc7yw/sHomnEy3STpNxXEmw7015xlBtwZyzjj7ajU+f2vO86yoao3gxltOeGp9uresz5sjL7/WJ+i32kQkDNgGDATScZaxHqGqqV5tYoE5wABVPSYijVX1kIg0AFKABJyEtBro7rapq6onxbkh+SHwgaq+5yae06r6fIE4OuAsh50IRANfAnGqmldU7OUy8eTnw67FTu9my2c2SaepHPJynV6R9wi6o2nOvirh0PRqr2dFCdYrKoan1ie2SW3mPdjnsm67hcKttkRgh6qmuQG9BwwFUr3ajAEmq+oxAFU95G6/HkhS1aPusUnAYGC2qp70ir0aTmIqzlDgPVXNBnaJyA43thWlvL7QcOaIMyotZQYc2+VUk/d60CbpNJVDWDg06+y8etzrbDtzxOtZ0Sqn95/8urOvdpOLR9BZrwiAmzo3I6p2NU5k5vh9Xjd/Jp7mwB6v9+nATwq0iQMQkWU4t+MmqOrnRRz74/zeIrIQJ3kswOn1eDwoInfh9JZ+4ya05sDKos7ldc6xwFiAli1b+nyRQaEKP6xwejepH5+fpHPAEzZJpzG1ouCqG5wXuL2iTRc+K9oy39l3Ua+ohzPwppL2in7SpmFAPsefiaewf7mCvZNwIBboB8QAS0SkU0nHqur1IhIBvAMMAJKA14Cn3XZPA5OAu32MA1WdCkwF51ZbMdcVPJnHYcP7TsI5vMUm6TTGF2Hh0KyL80oc42w7fdjrWVERvSLPwIXoeOsVlTF/Jp50oIXX+xhgXyFtVqpqDs5tsK04iSgdJxl5H7vY+0BVzRKReTi30pJU9aBnn4i8Acy/hDhClyrscyfp/M5rks6hk22STmMuV+1G0O5G5wVOr+jgxguHc//YK6rq9Iq8By7Ua1Fpe0VlwZ+DC8JxBhdcB+zFGVzwC1Xd5NVmMM6Ag5EiEgWsBeI5P6Cgm9t0DdAdOAfUUdX97vnfAZao6isi0kxV97vnfRT4iaoOF5GOwLucH1zwHyA25AcXZJ+GjR86CWf/epuk05hAO33owmdFe9c4v/iBM1LUe+G8ZvFQNSK48YaAoA8uUNVcEXkQWIjz/Ga6qm4SkYlAiqrOc/cNEpFUnKHOj6lqhnsBT+MkK4CJqnpURJoA80SkunvOr4Apbpu/iognae0G7nPj2CQic3AGNeTiDr/213WX2oGNzmqe698/P0nnTZNskk5jAq12Y2h3k/MCyMtxekU/Lie+CjZ/4uyrUtUZ4OBd5FovxnpFRbCZCwoR8B5PTqYzSCBlOuz51ibpNKa8KK5XVKfZhc+KmnWp8L2ioPd4jA+O7HB6N+vecZYhbnglXP9nm6TTmPKipF7RnmRneXhwe0VdCjwrqpwF3dbjKYRfezw2SacxlcupgxeOoNu3BnKznH11ogs8K+pSrsshrMcTagqbpPO6P9kkncZUdHWaQPshzgucXtGB7y4cQZf6sbMvrJqTfH58VtSjQvaKrMdTiDLr8dgkncYYX5w6eP7WXPoq2Le2XPaKrMcTTIVN0vnT39kkncaYwtVp4txyb3+z8z73HBz8zutZ0apiekWJUO+iyVhCmvV4CnHZPZ6T+2DB72ySTmNM2Tt14PytuT1urygv29lXt3mBEXSdg9Irsh5PMEREwsFUm6TTGFP26jSFDrc4L3B6RQe+u/AWXepcZ19Y9YtH0NWNDl7sBViPpxClesajaiPTjDHBcXJ/gWdF67x6RTEXPitq2hnCL2/dnaJYjydYLOkYY4KlbjPoMNR5gdsr2nD+Fl16Cmz6yNkXVt2Zfsv7Fl3dZgEJ03o8hQiJudqMMcYfTu67cLYF715RvRbOM+mbni/+HEWwHo8xxpiL1Y0u0CvKdp4VeXpF+Tl+D8ESjzHGVGbh1Z1lwWMSgPsD8pFVAvIpxhhjjMsSjzHGmICyxGOMMSag/Jp4RGSwiGwVkR0i8vsi2gwTkVQR2SQi73ptHyki293XSK/tn4vIerf9FBEJc7f/TUS2iMgGEflIRCLd7a1EJFNE1rmvKRdHYYwxJlD8lnjchDAZuAHoAIwQkQ4F2sQCjwO9VbUj8Ii7vQHwFPATnCWrnxKR+u5hw1S1C9AJaATc7m5PAjqpamecJbcf9/qonaoa777Glf3VGmOM8ZU/ezyJwA5VTVPVc8B7wNACbcYAk1X1GICqHnK3Xw8kqepRd18SMNhtc9JtEw5Uw1nqGlX9QlVz3X0rAZuN0xhjQpA/E09zYI/X+3R3m7c4IE5ElonIShEZ7MuxIrIQOAScAj4s5LPvBhZ4vW8tImtF5GsR6XtZV2OMMaZM+DPxFDZ3TMFpEsKBWKAfMAJ40302U+yxqno90AyoDgy44ENF/gjkAu+4m/YDLVW1KzAeeFdE6l4UrMhYEUkRkZTDhw+XfHXGGGMuiz8LSNOBFl7vY4B9hbRZqao5wC4R2YqTiNJxkpH3sYu9D1TVLBGZh3P7LgmcAQnAEOA6decCUtVsINv9erWI7MTpaaUUON9UYKp7nsMi8v1lXbV/RAFHgh1ECUI9xlCPD0I/xlCPD0I/xlCPD0oX4xU+tVJVv7xwkloa0BrnWcx6oGOBNoOBme7XUTi31xoCDYBdQH33tcvdVhto5nX+94EHvc6VCjQq8BmNgDD36zbAXqCBv67bT3+XKcGOobzHGOrxlYcYQz2+8hBjqMcXqBj91uNR1VwReRBYCIQB01V1k4hMdC9snrtvkIikAnnAY6qaASAiTwOr3NNNVNWjItIEmCci1d1zfgV4hke/gnPrLUmcGaJXqjOC7Vpgoojkup8xTlWP+uu6jTHGFM9mpy4HRCRFfZjxNZhCPcZQjw9CP8ZQjw9CP8ZQjw8CE6PNXFA+TA12AD4I9RhDPT4I/RhDPT4I/RhDPT4IQIzW4zHGGBNQ1uMxxhgTUJZ4Qkhp5rYLhfhEpKWILHKLdTeIyI0Bjm+6iBwSkY1F7BcR+Ycb/wYR6RbI+HyM8ZdubBtEZLmIdAml+Lza9RCRPBG5LVCxeX12iTGKSD93bsZNIvJ1IONzP7+kf+d6IvKJ17yTowMcXwv3e3Wz+/kPF9LGf98vwR66Z68fhzCGATtxhnx7hp93KNAmFlgL1HffNw6x+KYC/+1+3QHYHeC/w2uBbsDGIvbfiDOjhQDXAN8G4d+5pBh7ef373hDoGEuKz+v/wlfAZ8BtIfh3GIlTWtHSfR+w75NLiPEPwF/crxsBR4FqAYyvGdDN/boOzvyWBb+f/fb9Yj2e0FGaue1CJT4FPLNC1OPigmG/UtVvcL6BizIUmKWOlUCkiDQLTHSOkmJU1eWef1+CMOegD3+HAA8B/8KZtirgfIjxF8C/VfUHt33A4/QhRgXqiFP7Udttm1tM+zKlqvtVdY379SlgMxdPaea37xdLPKGjNHPbBYIv8U0AfiUi6Ti/DT8UmNB85ss1hJJ7uHDOwaATkebAzzlfPxeK4oD6IrJYRFaLyF3BDqgQrwDtcX45+w54WFXzgxGIiLQCugLfFtjlt+8Xf06ZYy7Npc5tFwMsEZFOqnrcz7GBb/GNAN5S1Uki0hN4240vKN9QhfDlGkKCiPTHSTx9gh1LAS8C/6OqeW6hdigKB7oD1wE1gBUislJVtwU3rAtcD6zDmWuyLU7h+xI9P/t+QIhIbZze6yOFfLbfvl8s8YSO0sxttwr/8yW+ezi/fMUKEYnAmQopKLdkCuHLNQSdiHQG3gRuUHcmjxCSALznJp0o4EYRyVXVucEN6wLpwBFVPQOcEZFvgC44zzFCxWjgOXUepuwQkV1AOyA5UAGISFWcpPOOqv67kCZ++36xW22hYxUQKyKtRaQaMByYV6DNXKA/gIhE4dxSSAuh+H7A+S0TEWkPRAChNNX3POAud7TONcAJVd0f7KC8iUhL4N/AnSH2GzoAqtpaVVupaiucJUnuD7GkA/Ax0FdEwkWkJs6CkpuDHFNB3t8rTYCrCNz3Mu6zpWnAZlV9oYhmfvt+sR5PiNBSzm0XIvH9BnhDRB7F6ZKPcn+jCwgRmY1zGzLKfc70FFDVjX8KznOnG4EdwFmc3zoDyocY/4QzUe6rbq8iVwM4xYoP8QVdSTGq6mYR+RzYAOQDb6pqscPDAx0j8DTwloh8h3NL639UNZCzVvcG7gS+E5F17rY/AC29YvTb94vNXGCMMSag7FabMcaYgLLEY4wxJqAs8RhjjAkoSzzGGGMCyhKPMcaYgLLEY4wxJqAs8RhTjrnT/88vbRtjAskSjzF+5FZ92/eZMV7sG8KYMiYirdwFtl4F1gB3isgKEVkjIh+4EzMiIjeKyBYRWeouuFVkr0REEsVZGG6t++dVhbSZICJvi8hXIrJdRMZ47a4tIh+6n/eOO2UKIvInEVklIhtFZKpnuzH+ZInHGP+4CpgFDMSZPPVnqtoNSAHGuxOovo4zEWgfnMXAirMFuFZVu+JMq/PnItp1Bm4CegJ/EpFod3tX4BGcBfra4EyZAvCKqvZQ1U44MzkPueQrNeYS2VxtxvjH96q6UkSG4PywX+Z2JqoBK3BmIk5T1V1u+9nA2GLOVw+YKSKxOPPgVS2i3ceqmglkisginAX8jgPJqpoO4M7N1QpYCvQXkd8BNYEGwCbgk8u7ZGN8Y4nHGP844/4pQJKqjvDeKSJdL/F8TwOLVPXn7sJdi4toV3DyRc/7bK9teUC42+t6FUhQ1T0iMgFnRnFj/MputRnjXyuB3iJyJYCI1BSROJxbZ23cJAJwRwnnqQfsdb8eVUy7oSISISINcWZHLm6tJk+SOeI+d7qthBiMKROWeIzxI1U9jJMoZovIBpxE1M69HXY/8LmILAUOAieKOdVfgWdFZBnOshRFSQY+dT/naVUtcuEud+XaN3CWXp5LYBYUNMaWRTAmWESktqqedkeSTQa2q+rfS3G+CcBpVX2+rGI0xh+sx2NM8IxxH/RvwrmV9nqQ4zEmIKzHY0wIEZHRwMMFNi9T1QeCEY8x/mCJxxhjTEDZrTZjjDEBZYnHGGNMQFniMcYYE1CWeIwxxgSUJR5jjDEB9f8BRqb3L9SZ1QYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch5_1.best_score_, gsearch5_1.best_params_))\n",
    "test_means = gsearch5_1.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch5_1.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch5_1.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch5_1.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch5_1.cv_results_).to_csv('my_preds_reg_alpha_reg_lambda_1.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(reg_alpha), len(reg_lambda))\n",
    "train_scores = np.array(train_means).reshape(len(reg_alpha), len(reg_lambda))\n",
    "\n",
    "#log_reg_alpha = [0,0,0,0]\n",
    "#for index in range(len(reg_alpha)):\n",
    "#   log_reg_alpha[index] = math.log10(reg_alpha[index])\n",
    "    \n",
    "for i, value in enumerate(reg_alpha):\n",
    "    pyplot.plot(reg_lambda, -test_scores[i], label= 'reg_alpha:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'reg_alpha' )                                                                                                      \n",
    "pyplot.ylabel( '-Log Loss' )\n",
    "pyplot.savefig( 'reg_alpha_vs_reg_lambda1.png' )"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
